{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from collections import defaultdict\n",
    "import random\n",
    "import json\n",
    "import copy\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.8/site-packages/malaya/tokenizer.py:202: FutureWarning: Possible nested set at position 3361\n",
      "  self.tok = re.compile(r'({})'.format('|'.join(pipeline)))\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/malaya/tokenizer.py:202: FutureWarning: Possible nested set at position 3879\n",
      "  self.tok = re.compile(r'({})'.format('|'.join(pipeline)))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ce5f7a1909041afb6f51ec4ad530a1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/480M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6816674d6ba47a4b9605c081a7dfa01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/569k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba7ed614c2744c8badd64eb1e38fd24a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/779k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-14 20:32:07.631005: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-11-14 20:32:07.670005: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:07.702551: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:07.703520: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:07.993175: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:07.993940: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:07.994667: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:07.995631: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
      "2022-11-14 20:32:07.995662: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /device:GPU:0 with 22047 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3090, pci bus id: 0000:01:00.0, compute capability: 8.6\n",
      "2022-11-14 20:32:09.884231: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:09.885232: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:09.885987: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:09.886729: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:09.887433: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:09.888119: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 22047 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3090, pci bus id: 0000:01:00.0, compute capability: 8.6\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8574f6e85bf34204b7fe5acc90c7f940",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/468M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-14 20:32:48.522671: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:48.523528: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:48.524270: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:48.525035: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:48.525784: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-11-14 20:32:48.526490: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 22047 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3090, pci bus id: 0000:01:00.0, compute capability: 8.6\n"
     ]
    }
   ],
   "source": [
    "import malaya\n",
    "from malaya.text.regex import _expressions\n",
    "\n",
    "model = malaya.dependency.transformer(model = 'xlnet')\n",
    "pos = malaya.pos.transformer(model = 'xlnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = malaya.tokenizer.Tokenizer(date = False, time = False).tokenize\n",
    "sastrawi = malaya.stem.sastrawi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/dictionary/synonym/synonym0.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/dictionary/synonym/synonym1.json\n",
    "files = ['synonym0.json', 'synonym1.json']\n",
    "synonyms = defaultdict(list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in files:\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    for i in data:\n",
    "        if not len(i[1]):\n",
    "            continue\n",
    "        synonyms[i[0]].extend(i[1])\n",
    "        for r in i[1]:\n",
    "            synonyms[r].append(i[0])\n",
    "            \n",
    "for k, v in synonyms.items():\n",
    "    synonyms[k] = list(set(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_t(tokens):\n",
    "    t = []\n",
    "    for i in range(len(tokens)):\n",
    "        t.append([tokens[i], 2])\n",
    "    return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_3_0(t, row, selected = ['compound', 'flat']):\n",
    "    text, tokens, tokens_lower, graph = row\n",
    "    l = list(graph.nodes.items())\n",
    "    for no, n in enumerate(l[1:]):\n",
    "        n = n[1]\n",
    "        if n['rel'] in selected and n['address'] - 1 == n['head']:\n",
    "            if n['word'] == t[n['head'] - 1][0]:\n",
    "                print('repeated word, continue')\n",
    "                continue\n",
    "            if n['word'][0].isupper() or t[n['head'] - 1][0][0].isupper():\n",
    "                continue\n",
    "            if n['word'].lower() in set_combined_penjodoh_bilangan or \\\n",
    "            t[n['head'] - 1][0].lower() in set_combined_penjodoh_bilangan:\n",
    "                continue\n",
    "                \n",
    "            c = t[n['head'] - 1].copy()\n",
    "            c[1] = 3\n",
    "            t[n['head'] - 1] = [t[n['address'] - 1][0], 3]\n",
    "            t[n['address'] - 1] = c\n",
    "            tokens[n['head'] - 1] = t[n['address'] - 1][0]\n",
    "            tokens[n['address'] - 1] = c[0]\n",
    "            tokens_lower[n['head'] - 1] = t[n['address'] - 1][0].lower()\n",
    "            tokens_lower[n['address'] - 1] = c[0].lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://ms.wikipedia.org/wiki/Penjodoh_bilangan_bahasa_Melayu\n",
    "penjodoh_bilangan = [\n",
    "    'angkatan',\n",
    "    'baris',\n",
    "    'batang',\n",
    "    'bentuk',\n",
    "    'bidang',\n",
    "    'biji',\n",
    "    'bilah',\n",
    "    'buah',\n",
    "    'buku',\n",
    "    'bungkus',\n",
    "    'butir',\n",
    "    'carik',\n",
    "    'cebis',\n",
    "    'cekak',\n",
    "    'cubit',\n",
    "    'cucuk',\n",
    "    'das',\n",
    "    'deret',\n",
    "    'ekor',\n",
    "    'gugus',\n",
    "    'gelung',\n",
    "    'gemal',\n",
    "    'genggam',\n",
    "    'gulung',\n",
    "    'gumpal',\n",
    "    'helai',\n",
    "    'hidangan',\n",
    "    'hiris',\n",
    "    'ikat',\n",
    "    'jambak',\n",
    "    'jambangan',\n",
    "    'jemput',\n",
    "    'kaki',\n",
    "    'kalung',\n",
    "    'kandang',\n",
    "    'kapur',\n",
    "    'kawan',\n",
    "    'kelompok',\n",
    "    'kepal',\n",
    "    'keping',\n",
    "    'kepul',\n",
    "    'kerat',\n",
    "    'ketul',\n",
    "    'kotak',\n",
    "    'kuntum',\n",
    "    'laras',\n",
    "    'lembar',\n",
    "    'lingkar',\n",
    "    'longgok',\n",
    "    'naskhah',\n",
    "    'orang',\n",
    "    'papan',\n",
    "    'pasang',\n",
    "    'pasukan',\n",
    "    'patah',\n",
    "    'pintu',\n",
    "    'potong',\n",
    "    'pucuk',\n",
    "    'puntung',\n",
    "    'rangkap',\n",
    "    'rawan',\n",
    "    'ruas',\n",
    "    'rumpun',\n",
    "    'sikat',\n",
    "    'sisir',\n",
    "    'suap',\n",
    "    'tandan',\n",
    "    'tangkai',\n",
    "    'teguk',\n",
    "    'timbun',\n",
    "    'titik',\n",
    "    'tongkol',\n",
    "    'ulas',\n",
    "    'untai',\n",
    "    'urat',\n",
    "    'utas',\n",
    "]\n",
    "hubung_list = [\n",
    "    'agar',\n",
    "    'apabila',\n",
    "    'atau',\n",
    "    'bahawa',\n",
    "    'dan',\n",
    "    'hingga',\n",
    "    'jika',\n",
    "    'jikalau',\n",
    "    'kecuali',\n",
    "    'kerana',\n",
    "    'lalu',\n",
    "    'manakala',\n",
    "    'sambil',\n",
    "    'serta',\n",
    "    'semenjak',\n",
    "    'sementara',\n",
    "    'sungguhpun',\n",
    "    'supaya',\n",
    "    'walaupun',\n",
    "    'tetapi',\n",
    "    'berkenan',\n",
    "    'berkenaan',\n",
    "    'yang',\n",
    "    'juga',\n",
    "    'tersebut'\n",
    "]\n",
    "end_4 = ['nya']\n",
    "reserved_4 = ['mereka', 'pelajar', 'rakyat', 'penduduk', 'umat', 'kami', 'semua', 'kumpulan', 'para']\n",
    "start_4 = ['be', 'ber', 'ter', 'se']\n",
    "sepenjodoh_bilangan = [f'se{w}' for w in penjodoh_bilangan]\n",
    "set_sepenjodoh_bilangan = set(sepenjodoh_bilangan)\n",
    "set_penjodoh_bilangan = set(penjodoh_bilangan)\n",
    "set_reserved_4 = set(reserved_4)\n",
    "set_combined_penjodoh_bilangan = set_sepenjodoh_bilangan | set_penjodoh_bilangan\n",
    "\n",
    "# [penjodoh bilangan] [kata nama] -> [penjodoh bilangan] [kata nama - kata nama]\n",
    "# dua buah kereta -> dua buah kereta-kereta\n",
    "def augment_4_0(t, row):\n",
    "    text, tokens, tokens_lower, penjodoh = row\n",
    "    for word in penjodoh:\n",
    "        try:\n",
    "            i = tokens_lower.index(word) + 1\n",
    "            if tokens_lower[i] in hubung_list:\n",
    "                continue\n",
    "            if tokens[i][0].isupper():\n",
    "                continue\n",
    "            if tokens[i].endswith('nya'):\n",
    "                tokens[i] = tokens[i][:-3]\n",
    "                ends = 'nya'\n",
    "            else:\n",
    "                ends = ''\n",
    "            word = f'{tokens[i]}-{tokens[i]}{ends}'\n",
    "            t[i][0] = word\n",
    "            t[i][1] = 4\n",
    "            tokens[i] = word\n",
    "            tokens_lower[i] = word.lower()\n",
    "        except Exception as e:\n",
    "            print('augment_4_0', e)\n",
    "            pass\n",
    "\n",
    "# [kata nama - kata nama] -> [kata nama]\n",
    "# ayam-ayam itu -> ayam itu\n",
    "def augment_4_1(t, row):\n",
    "    text, tokens, tokens_lower, penjodoh = row\n",
    "    for no, word in enumerate(tokens):\n",
    "        if re.findall(_expressions['hypen'], word.lower()):\n",
    "            stemmed = sastrawi.stem(word)\n",
    "            if stemmed != word.split('-')[0]:\n",
    "                continue\n",
    "            if word[0].isupper():\n",
    "                continue\n",
    "            word = word.split('-')[0]\n",
    "            t[no][0] = word\n",
    "            t[no][1] = 4\n",
    "            tokens[no] = word\n",
    "            tokens_lower[no] = word.lower()\n",
    "            \n",
    "            \n",
    "def augment_4_2(t, row):\n",
    "    text, tokens, tokens_lower, penjodoh = row\n",
    "    for word in penjodoh:\n",
    "        try:\n",
    "            i = tokens_lower.index(word)\n",
    "            if tokens[i].endswith('nya'):\n",
    "                tokens[i] = tokens[i][:-3]\n",
    "                ends = 'nya'\n",
    "            else:\n",
    "                ends = ''\n",
    "            t[i][0] = f'{tokens[i]}-{tokens[i]}{ends}'\n",
    "            t[i][1] = 4\n",
    "            tokens[i] = word\n",
    "            tokens_lower[i] = word.lower()\n",
    "        except Exception as e:\n",
    "            print('augment_4_2', e)\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "penguat_list = [\n",
    "    'paling',\n",
    "    'agak',\n",
    "    'sungguh',\n",
    "    'amat',\n",
    "    'terlalu',\n",
    "    'nian',\n",
    "    'benar',\n",
    "    'paling',\n",
    "    'sangat'\n",
    "]\n",
    "end_penguat_list = ['sekali', 'sungguh', 'sangat']\n",
    "set_penguat_list = set(penguat_list)\n",
    "\n",
    "def augment_5_0(t, row):\n",
    "    text, tokens, tokens_lower, penguat = row\n",
    "    for word in penguat:\n",
    "        try:\n",
    "            i = tokens_lower.index(word) + 1\n",
    "            if tokens[i][0].isupper():\n",
    "                continue\n",
    "            ends = random.choice(end_penguat_list)\n",
    "            word = f'{tokens[i]} {ends}'\n",
    "            t[i][0] = word\n",
    "            t[i][1] = 5\n",
    "            tokens[i] = word\n",
    "            tokens_lower[i] = word.lower()\n",
    "        except Exception as e:\n",
    "            print('augmentation_5_0', e)\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_start_ter(word):\n",
    "    stemmed = sastrawi.stem(word)\n",
    "    if word.startswith('ter') and not stemmed.startswith('ter') and stemmed in word:\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "def augment_6_0(t, row):\n",
    "    text, tokens, tokens_lower, penguat = row\n",
    "    for i in range(len(tokens)):\n",
    "        if check_start_ter(tokens[i]):\n",
    "            ends = random.choice(end_penguat_list)\n",
    "            word = f'{tokens[i]} {ends}'\n",
    "            t[i][0] = word\n",
    "            t[i][1] = 6\n",
    "            tokens[i] = word\n",
    "            tokens_lower[i] = word.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "hubung_list = [\n",
    "    'agar',\n",
    "    'apabila',\n",
    "    'atau',\n",
    "    'bahawa',\n",
    "    'dan',\n",
    "    'hingga',\n",
    "    'jika',\n",
    "    'jikalau',\n",
    "    'kecuali',\n",
    "    'kerana',\n",
    "    'lalu',\n",
    "    'manakala',\n",
    "    'sambil',\n",
    "    'serta',\n",
    "    'semenjak',\n",
    "    'sementara',\n",
    "    'sungguhpun',\n",
    "    'supaya',\n",
    "    'walaupun',\n",
    "    'tetapi',\n",
    "    'berkenan',\n",
    "    'berkenaan',\n",
    "]\n",
    "set_hubung_list = set(hubung_list)\n",
    "\n",
    "def augment_7_0(t, row):\n",
    "    text, tokens, tokens_lower, hubung = row\n",
    "    for word in hubung:\n",
    "        i = tokens_lower.index(word)\n",
    "        negate = list(set_hubung_list - {word})\n",
    "        choice = random.choice(negate)\n",
    "        t[i][0] = choice\n",
    "        t[i][1] = 7\n",
    "        tokens[i] = choice\n",
    "        tokens_lower[i] = choice.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_8 = ['be', 'ber', 'ter', 'se']\n",
    "\n",
    "def check_bilangan(word):\n",
    "    if re.findall(_expressions['hypen'], word.lower()):\n",
    "        stemmed = sastrawi.stem(word)\n",
    "        splitted = word.split('-')\n",
    "        for s in start_8:\n",
    "            if word.startswith(s) and f'{s}{stemmed}' == splitted[0] and stemmed == splitted[1]:\n",
    "                return True\n",
    "    return False\n",
    "\n",
    "def augment_8_0(t, row):\n",
    "    text, tokens, tokens_lower = row\n",
    "    for i in range(len(tokens)):\n",
    "        if check_bilangan(tokens[i]):\n",
    "            word = tokens[i].split('-')[0]\n",
    "            t[i][0] = word\n",
    "            t[i][1] = 8\n",
    "            tokens[i] = word\n",
    "            tokens_lower[i] = word.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sendi_list = [\n",
    "    'akan',\n",
    "    'kepada',\n",
    "    'terhadap',\n",
    "    'bagi',\n",
    "    'untuk',\n",
    "    'dari',\n",
    "    'daripada',\n",
    "    'di',\n",
    "    'dengan',\n",
    "    'hingga',\n",
    "    'sampai',\n",
    "    'ke',\n",
    "    'kepada',\n",
    "    'oleh',\n",
    "    'pada',\n",
    "    'sejak',\n",
    "    'seperti',\n",
    "    'umpama',\n",
    "    'bak',\n",
    "    'tentang',\n",
    "    'laksanabagai',\n",
    "    'semenjak',\n",
    "    'dalam',\n",
    "    'antara',\n",
    "]\n",
    "set_sendi_list = set(sendi_list)\n",
    "\n",
    "def augment_9_0(t, row):\n",
    "    text, tokens, tokens_lower, sendi = row\n",
    "    for word in sendi:\n",
    "        i = tokens_lower.index(word)\n",
    "        negate = list(set_sendi_list - {word})\n",
    "        choice = random.choice(negate)\n",
    "        t[i][0] = choice\n",
    "        t[i][1] = 9\n",
    "        tokens[i] = choice\n",
    "        tokens_lower[i] = choice.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_10_0(t, row):\n",
    "    text, tokens, tokens_lower, penjodoh = row\n",
    "    for word in penjodoh:\n",
    "        try:\n",
    "            i = tokens_lower.index(word)\n",
    "            negate = list(set_penjodoh_bilangan - {word})\n",
    "            choice = random.choice(negate)\n",
    "            t[i][0] = choice\n",
    "            t[i][1] = 10\n",
    "            tokens[i] = choice\n",
    "            tokens_lower[i] = choice.lower()\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            pass\n",
    "        \n",
    "def augment_10_1(t, row):\n",
    "    text, tokens, tokens_lower, penjodoh = row\n",
    "    for word in penjodoh:\n",
    "        try:\n",
    "            i = tokens_lower.index(word)\n",
    "            negate = list(set_sepenjodoh_bilangan - {word})\n",
    "            choice = random.choice(negate)\n",
    "            t[i][0] = choice\n",
    "            t[i][1] = 10\n",
    "            tokens[i] = choice\n",
    "            tokens_lower[i] = choice.lower()\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "gantinama_list = [\n",
    "    'aku',\n",
    "    'saya',\n",
    "    'hamba',\n",
    "    'patik',\n",
    "    'beta',\n",
    "    'kami',\n",
    "    'kita',\n",
    "    'anda',\n",
    "    'awak',\n",
    "    'engkau',\n",
    "    'tuanku',\n",
    "    'kalian',\n",
    "    'kamu',\n",
    "    'baginda',\n",
    "    'beliau',\n",
    "    'mereka',\n",
    "    'ini',\n",
    "    'itu',\n",
    "    'sini',\n",
    "    'situ',\n",
    "    'sana',\n",
    "    'kini',\n",
    "    'dia',\n",
    "    'kau',\n",
    "]\n",
    "set_gantinama_list = set(gantinama_list)\n",
    "\n",
    "def augment_11_0(t, row):\n",
    "    text, tokens, tokens_lower, nama = row\n",
    "    for word in nama:\n",
    "        i = tokens_lower.index(word)\n",
    "        negate = list(set_gantinama_list - {word})\n",
    "        choice = random.choice(negate)\n",
    "        t[i][0] = choice\n",
    "        t[i][1] = 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_12_0(t, row):\n",
    "    text, tokens, tokens_lower, tagging = row\n",
    "    for i in range(len(tokens) - 2):\n",
    "        if tagging[i] == 'ADV' \\\n",
    "        and tagging[i + 1] in ['PRON', 'NOUN'] \\\n",
    "        and tagging[i + 2] in ['VERB', 'NOUN'] \\\n",
    "        and tokens_lower[i] in ['telah', 'mesti']:\n",
    "            v = f'di{tokens[i + 2]}'\n",
    "            n = f'oleh {tokens[i + 1]}'\n",
    "            t[i][1] = 12\n",
    "            t[i + 1][0] = v\n",
    "            t[i + 1][1] = 12\n",
    "            t[i + 2][0] = n\n",
    "            t[i + 2][1] = 12\n",
    "            \n",
    "def augment_12_1(t, row):\n",
    "    text, tokens, tokens_lower, tagging = row\n",
    "    for i in range(len(tokens) - 1):\n",
    "        if tagging[i] == 'PRON' and tagging[i + 1] == 'VERB' and sastrawi.stem(tokens[i + 1]) == tokens[i + 1]:\n",
    "            v = f'men{tokens[i + 1]}'\n",
    "            if sastrawi.stem(v) == v:\n",
    "                v = f'mem{tokens[i + 1]}'\n",
    "            t[i][1] = 12\n",
    "            t[i + 1][0] = v\n",
    "            t[i + 1][1] = 12\n",
    "            \n",
    "def augment_12_2(t, row):\n",
    "    text, tokens, tokens_lower, tagging = row\n",
    "    for i in range(len(tokens) - 2):\n",
    "        if tagging[i] == 'VERB' \\\n",
    "        and tagging[i + 1] in ['ADP'] \\\n",
    "        and tagging[i + 2] in ['PRON', 'NOUN'] \\\n",
    "        and tokens_lower[i + 1] in ['oleh']:\n",
    "            v = sastrawi.stem(tokens[i])\n",
    "            t[i][0] = tokens[i + 2]\n",
    "            t[i][1] = 12\n",
    "            t[i + 1][0] = v\n",
    "            t[i + 1][1] = 12\n",
    "            t[i + 2][0] = ''\n",
    "            t[i + 2][1] = 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tanya_list = [\n",
    "    'kenapa',\n",
    "    'bila',\n",
    "    'siapa',\n",
    "    'mengapa',\n",
    "    'apa',\n",
    "    'bagaimana',\n",
    "    'berapa',\n",
    "    'mana',\n",
    "]\n",
    "kah_tanya_list = [f'{w}kah' for w in tanya_list]\n",
    "combined = tanya_list + kah_tanya_list\n",
    "set_combined = set(combined)\n",
    "\n",
    "def augment_13_0(t, row):\n",
    "    text, tokens, tokens_lower, tanya = row\n",
    "    for word in tanya:\n",
    "        i = tokens_lower.index(word)\n",
    "        negate = list(set_combined - {word})\n",
    "        choice = random.choice(negate)\n",
    "        t[i][0] = choice\n",
    "        t[i][1] = 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "punc = '.?!,;:'\n",
    "set_punc = set(punc)\n",
    "\n",
    "def augment_14_0(t, row):\n",
    "    text, tokens, tokens_lower, p = row\n",
    "    for word in p:\n",
    "        i = tokens_lower.index(word)\n",
    "        negate = list(set_punc - {word})\n",
    "        choice = random.choice(negate)\n",
    "        t[i][0] = choice\n",
    "        t[i][1] = 14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_15 = ['ber', 'ter', 'me', 'men']\n",
    "\n",
    "def check_tak_transitif(word):\n",
    "    stemmed = sastrawi.stem(word)\n",
    "    for s in start_15:\n",
    "        if word.startswith(s) and f'{s}{stemmed}' == word:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "def augment_15_0(t, row):\n",
    "    text, tokens, tokens_lower, tagging = row\n",
    "    for i in range(len(tokens) - 1):\n",
    "        if tagging[i] == 'VERB' \\\n",
    "        and tagging[i + 1] not in ['PRON', 'NOUN'] \\\n",
    "        and check_tak_transitif(tokens[i]):\n",
    "            t[i][0] = sastrawi.stem(tokens[i])\n",
    "            t[i][1] = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_end = {'me': 'kan', 'mem': 'kan', 'men': 'kan', 'mem': '', 'me': '', 'men': ''}\n",
    "\n",
    "def check_transitif(word):\n",
    "    stemmed = sastrawi.stem(word)\n",
    "    for k, v in start_end.items():\n",
    "        if word.startswith(k) and word.endswith(v) and f'{k}{stemmed}{v}' == word:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "def augment_16_0(t, row):\n",
    "    text, tokens, tokens_lower, tagging = row\n",
    "    for i in range(len(tokens) - 2):\n",
    "        if tagging[i] in ['PRON', 'NOUN'] \\\n",
    "        and tagging[i + 1] == 'VERB' \\\n",
    "        and tagging[i + 2] in ['PRON', 'NOUN'] \\\n",
    "        and check_transitif(tokens[i + 1]):\n",
    "            t[i][1] = 16\n",
    "            t[i + 1][0] = sastrawi.stem(tokens[i + 1])\n",
    "            t[i + 1][1] = 16\n",
    "            t[i + 2][1] = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_17_0(t, row):\n",
    "    text, tokens, tokens_lower = row\n",
    "    for i in range(len(tokens)):\n",
    "        if tokens_lower[i] in synonyms:\n",
    "            w = random.choice(synonyms[tokens_lower[i]])\n",
    "            t[i][0] = w\n",
    "            t[i][1] = 17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/dumping/clean/filtered-dumping-wiki.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2037249"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('filtered-dumping-wiki.txt') as fopen:\n",
    "    data = list(filter(None, fopen.read().split('\\n')))\n",
    "    \n",
    "data = [i for i in data if len(i) >= 2]\n",
    "\n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "extra = [\n",
    "    'gunung itu sangat tinggi',\n",
    "    'dapat markah yang tertinggi',\n",
    "    'jom mengaji agar kita pandai',\n",
    "    'berjuta-juta rakyat malaysia',\n",
    "    'aku sayang akan engkau',\n",
    "    '2 buah kereta',\n",
    "    'jom mengaji agar kita pandai',\n",
    "    'Cerpen itu telah saya karang.', \n",
    "    'Latihan itu mesti kau buat.',\n",
    "    'Kereta itu saya beli daripada Ali.',\n",
    "    'Surat itu dihantar oleh abang semalam.',\n",
    "    'Kamu berasal dari mana?',\n",
    "    'jom mengaji agar kita pandai.',\n",
    "    'Cerpen itu telah saya karang.', \n",
    "    'Latihan itu mesti kau buat.',\n",
    "    'Kereta itu saya beli daripada Ali.',\n",
    "    'Surat itu dihantar oleh abang semalam.',\n",
    "    'Jangan melompat.'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                                      | 0/2037249 [00:00<?, ?it/s]2022-11-14 20:43:03.025599: I tensorflow/stream_executor/cuda/cuda_blas.cc:1760] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "2022-11-14 20:43:03.025628: I tensorflow/stream_executor/cuda/cuda_blas.cc:1760] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "  0%|                                                                                                          | 67/2037249 [00:15<76:56:56,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n",
      "repeated word, continue\n",
      "repeated word, continue\n",
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                          | 70/2037249 [00:15<67:18:46,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n",
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                          | 76/2037249 [00:16<73:40:53,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                          | 84/2037249 [00:17<71:52:17,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                         | 127/2037249 [00:22<76:59:30,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list index out of range\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                         | 162/2037249 [00:27<71:27:58,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list index out of range\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                         | 175/2037249 [00:29<88:15:46,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n",
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                         | 190/2037249 [00:31<64:15:33,  8.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list index out of range\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                         | 284/2037249 [00:42<77:50:19,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n",
      "repeated word, continue\n",
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                         | 310/2037249 [00:46<89:23:58,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n",
      "repeated word, continue\n",
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                         | 708/2037249 [01:32<67:29:40,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list index out of range\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                         | 745/2037249 [01:36<73:55:56,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                         | 904/2037249 [01:53<73:17:16,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n",
      "repeated word, continue\n",
      "repeated word, continue\n",
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                        | 1019/2037249 [02:06<57:04:13,  9.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list index out of range\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                        | 1030/2037249 [02:08<55:48:42, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                        | 1066/2037249 [02:12<58:57:42,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n",
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                        | 1104/2037249 [02:16<72:28:15,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n",
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                        | 1281/2037249 [02:39<58:45:21,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeated word, continue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                                        | 1342/2037249 [02:45<48:58:14, 11.55it/s]"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "threshold = 0.5\n",
    "for text in tqdm(data):\n",
    "    try:\n",
    "        tokens = tokenizer(text)\n",
    "        t = reset_t(tokens)\n",
    "        t_ = copy.deepcopy(t)\n",
    "        tokens_lower = tokenizer(text.lower())\n",
    "        tagging, indexing = malaya.stack.voting_stack([model] * 3, ' '.join(tokens))\n",
    "        graph = malaya.dependency.dependency_graph(tagging, indexing)\n",
    "\n",
    "        pos_tagging = malaya.stack.voting_stack([pos] * 3, ' '.join(tokens))\n",
    "        pos_tagging = list(zip(*pos_tagging))[1]\n",
    "\n",
    "        r = (t, tokens, tokens_lower, graph)\n",
    "        if random.random() > threshold:\n",
    "            augment_3_0(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r_penjodoh_bilangan = set_tokens & set_penjodoh_bilangan\n",
    "        r_sepenjodoh_bilangan = set_tokens & set_sepenjodoh_bilangan\n",
    "        r_reserved = set_tokens & set_reserved_4\n",
    "        r = (t, tokens, tokens_lower, r_penjodoh_bilangan | r_sepenjodoh_bilangan)\n",
    "        \n",
    "        if random.random() > threshold:\n",
    "            augment_4_1(t_, r)\n",
    "        if random.random() > threshold:\n",
    "            augment_4_0(t_, r)\n",
    "\n",
    "        r = (t, tokens, tokens_lower, r_reserved)\n",
    "        if random.random() > threshold:\n",
    "            augment_4_2(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r_penguat_list = set_tokens & set_penguat_list\n",
    "        r = (t, tokens, tokens_lower, r_penguat_list)\n",
    "        if random.random() > threshold:\n",
    "            augment_5_0(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r_penguat_list = set_tokens & set_penguat_list\n",
    "        r = (t, tokens, tokens_lower, r_penguat_list)\n",
    "        if random.random() > threshold:\n",
    "            augment_6_0(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r_hubung_list = set_tokens & set_hubung_list\n",
    "        r = (t, tokens, tokens_lower, r_hubung_list)\n",
    "        if random.random() > threshold:\n",
    "            augment_7_0(t_, r)\n",
    "\n",
    "        r = (t, tokens, tokens_lower)\n",
    "        if random.random() > threshold:\n",
    "            augment_8_0(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r_sendi_list = set_tokens & set_sendi_list\n",
    "        r = (t, tokens, tokens_lower, r_sendi_list)\n",
    "        if random.random() > threshold:\n",
    "            augment_9_0(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r_penjodoh_bilangan = set_tokens & set_penjodoh_bilangan\n",
    "        r_sepenjodoh_bilangan = set_tokens & set_sepenjodoh_bilangan\n",
    "        r = (t, tokens, tokens_lower, r_penjodoh_bilangan)\n",
    "        if random.random() > threshold:\n",
    "            augment_10_0(t_, r)\n",
    "        r = (t, tokens, tokens_lower, r_sepenjodoh_bilangan)\n",
    "        if random.random() > threshold:\n",
    "            augment_10_1(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r_gantinama_list = set_tokens & set_gantinama_list\n",
    "        r = (t, tokens, tokens_lower, r_gantinama_list)\n",
    "        if random.random() > threshold:\n",
    "            augment_11_0(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r = (t, tokens, tokens_lower, pos_tagging)\n",
    "        if random.random() > threshold:\n",
    "            augment_12_0(t_, r)\n",
    "        a = list(zip(*t_))[1]\n",
    "        if 12 not in a:\n",
    "            if random.random() > threshold:\n",
    "                augment_12_1(t_, r)\n",
    "        a = list(zip(*t_))[1]\n",
    "        if 12 not in a:\n",
    "            if random.random() > threshold:\n",
    "                augment_12_2(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r_set_combined = set_tokens & set_combined\n",
    "        r = (t, tokens, tokens_lower, r_set_combined)\n",
    "        if random.random() > threshold:\n",
    "            augment_13_0(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r_set_punc = set_tokens & set_punc\n",
    "        r = (t, tokens, tokens_lower, r_set_punc)\n",
    "        if random.random() > threshold:\n",
    "            augment_14_0(t_, r)\n",
    "\n",
    "        set_tokens = set(tokens_lower)\n",
    "        r = (t, tokens, tokens_lower, pos_tagging)\n",
    "        if random.random() > threshold:\n",
    "            augment_15_0(t_, r)\n",
    "\n",
    "        r = (t, tokens, tokens_lower, pos_tagging)\n",
    "        if random.random() > threshold:\n",
    "            augment_16_0(t_, r)\n",
    "\n",
    "        r = (t, tokens, tokens_lower)\n",
    "        if random.random() > 0.8:\n",
    "            augment_17_0(t_, r)\n",
    "\n",
    "        results.append((t, t_))\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('dataset-tatabahasa.pkl', 'wb') as fopen:\n",
    "    pickle.dump(results, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('dataset-tatabahasa.pkl', 'rb') as fopen:\n",
    "    data = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "497079"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([['Erin', 2],\n",
       "  ['Grey', 2],\n",
       "  ['Van', 2],\n",
       "  ['Oosbree', 2],\n",
       "  ['(', 2],\n",
       "  ['lahir', 2],\n",
       "  ['24', 2],\n",
       "  ['Ogos', 2],\n",
       "  ['1973', 2],\n",
       "  [')', 2],\n",
       "  [',', 2],\n",
       "  ['dikreditkan', 2],\n",
       "  ['sebagai', 2],\n",
       "  ['Grey', 2],\n",
       "  ['DeLisle', 2],\n",
       "  ['(', 2],\n",
       "  [')', 2],\n",
       "  ['dan', 2],\n",
       "  ['Gray', 2],\n",
       "  ['Griffin', 2],\n",
       "  [',', 2],\n",
       "  ['ialah', 2],\n",
       "  ['seorang', 2],\n",
       "  ['pelakon', 2],\n",
       "  ['suara', 2],\n",
       "  [',', 2],\n",
       "  ['pelawak', 2],\n",
       "  ['dan', 2],\n",
       "  ['penyanyi-penulis', 2],\n",
       "  ['lagu', 2],\n",
       "  ['Amerika', 2],\n",
       "  ['Syarikat', 2],\n",
       "  ['.', 2]],\n",
       " [['Erin', 2],\n",
       "  ['Grey', 2],\n",
       "  ['Van', 2],\n",
       "  ['Oosbree', 2],\n",
       "  ['(', 2],\n",
       "  ['lahir', 2],\n",
       "  ['24', 2],\n",
       "  ['Ogos', 2],\n",
       "  ['1973', 2],\n",
       "  [')', 2],\n",
       "  [':', 14],\n",
       "  ['dikreditkan', 2],\n",
       "  ['sebagai', 2],\n",
       "  ['Grey', 2],\n",
       "  ['DeLisle', 2],\n",
       "  ['(', 2],\n",
       "  [')', 2],\n",
       "  ['dan', 2],\n",
       "  ['Gray', 2],\n",
       "  ['Griffin', 2],\n",
       "  [',', 2],\n",
       "  ['ialah', 2],\n",
       "  ['segelung', 10],\n",
       "  ['pelakon', 2],\n",
       "  ['suara', 2],\n",
       "  [',', 2],\n",
       "  ['pelawak', 2],\n",
       "  ['dan', 2],\n",
       "  ['penyanyi-penulis', 2],\n",
       "  ['lagu', 2],\n",
       "  ['Amerika', 2],\n",
       "  ['Syarikat', 2],\n",
       "  [',', 14]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 497079/497079 [00:04<00:00, 108093.60it/s]\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from tqdm import tqdm\n",
    "\n",
    "with open('train2.jsonl', 'w') as fopen:\n",
    "    for d in tqdm(data):\n",
    "        fopen.write(f'{json.dumps(d)}\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
